{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "solving pendulum using actor-critic model\n",
    "\"\"\"\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Input\n",
    "from tensorflow.keras.layers import Add, Concatenate\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import random\n",
    "from collections import deque\n",
    "\n",
    "def stack_samples(samples):\n",
    "\tarray = np.array(samples)\n",
    "\t\n",
    "\tcurrent_states = np.stack(array[:,0]).reshape((array.shape[0],-1))\n",
    "\tactions = np.stack(array[:,1]).reshape((array.shape[0],-1))\n",
    "\trewards = np.stack(array[:,2]).reshape((array.shape[0],-1))\n",
    "\tnew_states = np.stack(array[:,3]).reshape((array.shape[0],-1))\n",
    "\tdones = np.stack(array[:,4]).reshape((array.shape[0],-1))\n",
    "\t\n",
    "\treturn current_states, actions, rewards, new_states, dones\n",
    "\t\n",
    "\n",
    "# determines how to assign values to each state, i.e. takes the state\n",
    "# and action (two-input model) and determines the corresponding value\n",
    "class ActorCritic:\n",
    "\tdef __init__(self, env, sess):\n",
    "\t\tself.env  = env\n",
    "\t\tself.sess = sess\n",
    "\n",
    "\t\tself.learning_rate = 0.0001\n",
    "\t\tself.epsilon = .9\n",
    "\t\tself.epsilon_decay = .99995\n",
    "\t\tself.gamma = .90\n",
    "\t\tself.tau   = .01\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                               Actor Model                             #\n",
    "\t\t# Chain rule: find the gradient of chaging the actor network params in  #\n",
    "\t\t# getting closest to the final value network predictions, i.e. de/dA    #\n",
    "\t\t# Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act #\n",
    "\t\t# ===================================================================== #\n",
    "\t\tself.ema = tf.train.ExponentialMovingAverage(decay=1-self.tau)\n",
    "\t\tself.memory = deque(maxlen=4000)\n",
    "\t\tself.actor_state_input, self.actor_model = self.create_actor_model()\n",
    "\t\t_, self.target_actor_model = self.create_actor_model()\n",
    "\n",
    "\t\tself.actor_critic_grad = tf.placeholder(tf.float32,\n",
    "\t\t\t[None, self.env.action_space.shape[0]]) # where we will feed de/dC (from critic)\n",
    "\n",
    "\t\tactor_model_weights = self.actor_model.trainable_weights\n",
    "\t\tself.actor_grads = tf.gradients(self.actor_model.output,\n",
    "\t\t\tactor_model_weights, -self.actor_critic_grad) # dC/dA (from actor)\n",
    "\t\tgrads = zip(self.actor_grads, actor_model_weights)\n",
    "\t\tself.optimize = tf.train.AdamOptimizer(self.learning_rate).apply_gradients(grads)\n",
    "\n",
    "        \n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                              Critic Model                             #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.critic_state_input, self.critic_action_input, \\\n",
    "\t\t\tself.critic_model = self.create_critic_model()\n",
    "\t\t_, _, self.target_critic_model = self.create_critic_model()\n",
    "\n",
    "\t\tself.critic_grads = tf.gradients(self.critic_model.output,\n",
    "\t\t\tself.critic_action_input) # where we calcaulte de/dC for feeding above\n",
    "\n",
    "\t\t# Initialize for later gradient calculations\n",
    "\t\tself.sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "\t\tself.target_actor_upd = tf.group([tf.keras.backend.moving_average_update(self.target_actor_model.weights[i], \n",
    "                                                                         self.actor_model.get_weights()[i],\n",
    "                                                                        self.tau) for i in range(len(self.target_actor_model.weights))])\n",
    "\t\tself.target_critic_upd = tf.group([tf.keras.backend.moving_average_update(self.target_critic_model.weights[i], \n",
    "                                                                         self.critic_model.get_weights()[i],\n",
    "                                                                        self.tau) for i in range(len(self.target_critic_model.weights))])\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Definitions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef create_actor_model(self):\n",
    "\t\tstate_input = Input(shape=self.env.observation_space.shape)\n",
    "\t\th1 = Dense(500, activation='relu')(state_input)\n",
    "\t\th2 = Dense(1000, activation='relu')(h1)\n",
    "\t\th3 = Dense(500, activation='relu')(h2)\n",
    "\t\toutput = Dense(self.env.action_space.shape[0], activation='tanh')(h3)\n",
    "\n",
    "\t\tmodel = Model([state_input], output)\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, model\n",
    "\n",
    "\tdef create_critic_model(self):\n",
    "\t\tstate_input = Input(shape=self.env.observation_space.shape)\n",
    "\t\tstate_h1 = Dense(500, activation='relu')(state_input)\n",
    "\t\tstate_h2 = Dense(1000)(state_h1)\n",
    "\n",
    "\t\taction_input = Input(shape=self.env.action_space.shape)\n",
    "\t\taction_h1    = Dense(500)(action_input)\n",
    "\n",
    "\t\tmerged    = Concatenate()([state_h2, action_h1])\n",
    "\t\tmerged_h1 = Dense(500, activation='relu')(merged)\n",
    "\t\toutput = Dense(1, activation='linear')(merged_h1)\n",
    "\t\tmodel  = Model([state_input,action_input],output)\n",
    "\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, action_input, model\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                               Model Training                              #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef remember(self, cur_state, action, reward, new_state, done):\n",
    "\t\tself.memory.append([cur_state, action, reward, new_state, done])\n",
    "\n",
    "\tdef _train_actor(self, samples):\n",
    "\t\t\n",
    "\t\t\tcur_states, actions, rewards, new_states, _ =  stack_samples(samples)\n",
    "\t\t\tpredicted_actions = self.actor_model.predict(cur_states)\n",
    "\t\t\tgrads = self.sess.run(self.critic_grads, feed_dict={\n",
    "\t\t\t\tself.critic_state_input:  cur_states,\n",
    "\t\t\t\tself.critic_action_input: predicted_actions\n",
    "\t\t\t})[0]\n",
    "\n",
    "\t\t\tself.sess.run(self.optimize, feed_dict={\n",
    "\t\t\t\tself.actor_state_input: cur_states,\n",
    "\t\t\t\tself.actor_critic_grad: grads\n",
    "\t\t\t})\n",
    "\n",
    "\tdef _train_critic(self, samples):\n",
    "   \n",
    "\n",
    "\t\tcur_states, actions, rewards, new_states, dones = stack_samples(samples)\n",
    "\t\ttarget_actions = self.target_actor_model.predict(new_states)\n",
    "\t\tfuture_rewards = self.target_critic_model.predict([new_states, target_actions])\n",
    "\t\t\n",
    "\t\trewards += self.gamma * future_rewards * (1 - dones)\n",
    "\t\t\n",
    "\t\tevaluation = self.critic_model.fit([cur_states, actions], rewards, verbose=0)\n",
    "\t\t#print(evaluation.history)\n",
    "\tdef train(self):\n",
    "\t\tbatch_size = 256\n",
    "\t\tif len(self.memory) < batch_size:\n",
    "\t\t\treturn\n",
    "\n",
    "\t\trewards = []\n",
    "\t\tsamples = random.sample(self.memory, batch_size)\n",
    "\t\tself.samples = samples\n",
    "\t\tself._train_critic(samples)\n",
    "\t\tself._train_actor(samples)\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                         Target Model Updating                             #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef _update_actor_target(self):\n",
    "\t\tself.sess.run(self.target_actor_upd)\n",
    "        \n",
    "\tdef _update_critic_target(self):\n",
    "\t\tself.sess.run(self.target_critic_upd)\n",
    "\t\t#self.target_actor_model.set_weights(ema.average(critic_target_weights))\n",
    "        \n",
    "\tdef update_target(self):\n",
    "\t\tself._update_actor_target()\n",
    "\t\tself._update_critic_target()\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Predictions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef act(self, cur_state):\n",
    "\t\tself.epsilon *= self.epsilon_decay\n",
    "\t\tif np.random.random() < self.epsilon:\n",
    "\t\t\treturn self.actor_model.predict(cur_state)*2 + np.random.normal()\n",
    "\t\treturn self.actor_model.predict(cur_state)*2\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trial:0\n"
     ]
    },
    {
     "ename": "FailedPreconditionError",
     "evalue": "Attempting to use uninitialized value dense_7/bias/biased\n\t [[node dense_7/bias/biased/read (defined at <ipython-input-1-cd07e943b7c0>:80) ]]\n\nOriginal stack trace for 'dense_7/bias/biased/read':\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 664, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 583, in start\n    self.io_loop.start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 149, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 381, in dispatch_queue\n    yield self.process_one()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 225, in wrapper\n    runner = Runner(result, future, yielded)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 714, in __init__\n    self.run()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 545, in execute_request\n    user_expressions, allow_stdin,\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 300, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2858, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2886, in _run_cell\n    return runner(coro)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3063, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3254, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3331, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-2-924e6c052d18>\", line 61, in <module>\n    main()\n  File \"<ipython-input-2-924e6c052d18>\", line 5, in main\n    actor_critic = ActorCritic(env, sess)\n  File \"<ipython-input-1-cd07e943b7c0>\", line 80, in __init__\n    self.tau) for i in range(len(self.target_actor_model.weights))])\n  File \"<ipython-input-1-cd07e943b7c0>\", line 80, in <listcomp>\n    self.tau) for i in range(len(self.target_actor_model.weights))])\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\", line 1559, in moving_average_update\n    x, value, momentum, zero_debias=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 107, in assign_moving_average\n    return replica_context.merge_call(merge_fn, args=(variable, value))\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\", line 1684, in merge_call\n    return self._merge_call(merge_fn, args, kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\", line 1691, in _merge_call\n    return merge_fn(self._strategy, *args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 105, in merge_fn\n    return update(strategy, v, value)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 94, in update\n    return _zero_debias(strategy, v, value, decay)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 237, in _zero_debias\n    trainable=False)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 1496, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 1239, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 562, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 514, in _true_getter\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 929, in _get_single_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 259, in __call__\n    return cls._variable_v1_call(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 220, in _variable_v1_call\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 198, in <lambda>\n    previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 2511, in default_variable_creator\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 263, in __call__\n    return super(VariableMetaclass, cls).__call__(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 1568, in __init__\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 1755, in _init_from_args\n    self._snapshot = array_ops.identity(self._variable, name=\"read\")\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py\", line 180, in wrapper\n    return target(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\", line 86, in identity\n    ret = gen_array_ops.identity(input, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 4253, in identity\n    \"Identity\", input=input, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mFailedPreconditionError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFailedPreconditionError\u001b[0m: Attempting to use uninitialized value dense_7/bias/biased\n\t [[{{node dense_7/bias/biased/read}}]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mFailedPreconditionError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-924e6c052d18>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-924e6c052d18>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m                         \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-cd07e943b7c0>\u001b[0m in \u001b[0;36mupdate_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_actor_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_critic_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-cd07e943b7c0>\u001b[0m in \u001b[0;36m_update_actor_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_update_actor_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_actor_upd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_update_critic_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1368\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1369\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror_interpolation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1370\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1372\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFailedPreconditionError\u001b[0m: Attempting to use uninitialized value dense_7/bias/biased\n\t [[node dense_7/bias/biased/read (defined at <ipython-input-1-cd07e943b7c0>:80) ]]\n\nOriginal stack trace for 'dense_7/bias/biased/read':\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 664, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 583, in start\n    self.io_loop.start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 149, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 381, in dispatch_queue\n    yield self.process_one()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 225, in wrapper\n    runner = Runner(result, future, yielded)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 714, in __init__\n    self.run()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 545, in execute_request\n    user_expressions, allow_stdin,\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 300, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2858, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2886, in _run_cell\n    return runner(coro)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3063, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3254, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 3331, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-2-924e6c052d18>\", line 61, in <module>\n    main()\n  File \"<ipython-input-2-924e6c052d18>\", line 5, in main\n    actor_critic = ActorCritic(env, sess)\n  File \"<ipython-input-1-cd07e943b7c0>\", line 80, in __init__\n    self.tau) for i in range(len(self.target_actor_model.weights))])\n  File \"<ipython-input-1-cd07e943b7c0>\", line 80, in <listcomp>\n    self.tau) for i in range(len(self.target_actor_model.weights))])\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\", line 1559, in moving_average_update\n    x, value, momentum, zero_debias=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 107, in assign_moving_average\n    return replica_context.merge_call(merge_fn, args=(variable, value))\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\", line 1684, in merge_call\n    return self._merge_call(merge_fn, args, kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\", line 1691, in _merge_call\n    return merge_fn(self._strategy, *args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 105, in merge_fn\n    return update(strategy, v, value)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 94, in update\n    return _zero_debias(strategy, v, value, decay)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py\", line 237, in _zero_debias\n    trainable=False)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 1496, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 1239, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 562, in get_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 514, in _true_getter\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 929, in _get_single_variable\n    aggregation=aggregation)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 259, in __call__\n    return cls._variable_v1_call(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 220, in _variable_v1_call\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 198, in <lambda>\n    previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\", line 2511, in default_variable_creator\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 263, in __call__\n    return super(VariableMetaclass, cls).__call__(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 1568, in __init__\n    shape=shape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py\", line 1755, in _init_from_args\n    self._snapshot = array_ops.identity(self._variable, name=\"read\")\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py\", line 180, in wrapper\n    return target(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\", line 86, in identity\n    ret = gen_array_ops.identity(input, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 4253, in identity\n    \"Identity\", input=input, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "\tsess = tf.Session()\n",
    "\tK.set_session(sess)\n",
    "\tenv = gym.make(\"Pendulum-v0\")\n",
    "\tactor_critic = ActorCritic(env, sess)\n",
    "\n",
    "\tnum_trials = 10000\n",
    "\ttrial_len  = 200\n",
    "\n",
    "\tfor i in range(num_trials):\n",
    "\t\tprint(\"trial:\" + str(i))\n",
    "\t\tcur_state = env.reset()\n",
    "\t\taction = env.action_space.sample()\n",
    "\t\treward_sum = 0\n",
    "\t\tfor j in range(trial_len):\n",
    "\t\t\t#env.render()\n",
    "\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\taction = actor_critic.act(cur_state)\n",
    "\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\treward += reward\n",
    "\t\t\tif j == (trial_len - 1):\n",
    "\t\t\t\tdone = True\n",
    "\t\t\t\tprint(reward)\n",
    "\n",
    "\t\t\tif (j % 5 == 0):\n",
    "\t\t\t\tactor_critic.train()\n",
    "\t\t\t\tactor_critic.update_target()   \n",
    "\t\t\t\n",
    "\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "\t\t\tactor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "\t\t\tcur_state = new_state\n",
    "\n",
    "\t\tif (i % 5 == 0):\n",
    "\t\t\tcur_state = env.reset()\n",
    "\t\t\tfor j in range(500):\n",
    "\t\t\t\tenv.render()\n",
    "\t\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\t\taction = actor_critic.act(cur_state)\n",
    "\t\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\t\t#reward += reward\n",
    "\t\t\t\t#if j == (trial_len - 1):\n",
    "\t\t\t\t\t#done = True\n",
    "\t\t\t\t\t#print(reward)\n",
    "\n",
    "\t\t\t\t#if (j % 5 == 0):\n",
    "\t\t\t\t#    actor_critic.train()\n",
    "\t\t\t\t#    actor_critic.update_target()   \n",
    "\t\t\t\t\n",
    "\t\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "\t\t\t\t#actor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "\t\t\t\tcur_state = new_state\n",
    "\t\t\t\t\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
